fromFolder = "C:\Users\walsworthlab\Desktop\jstang\RandomCurrentGeneration\package\currentdensity_raw_test\";
toFolder = "C:\Users\walsworthlab\Desktop\jstang\RandomCurrentGeneration\package\currentdensity_raw_test_compressed\";
problematicls = [];
if ~exist(toFolder, 'dir')
   mkdir(toFolder)
end
dataFrom = 1;
dataTo = 10;
datals = linspace(dataFrom, dataTo, dataTo-dataFrom+1);
% load data
for index = datals
    flag = 0;
    disp("compressing")
    disp(index)
    dataindex = num2str(index,'%04.f');
    dataappendix = ".txt";
    fromcompletedatapath = fromFolder+dataindex+dataappendix;
    data = importdata(fromcompletedatapath);
    data(isnan(data)) = 0;
    if size(data,2) == 4
        data_simple = data(:,3:4);
        if size(data_simple,1) ~= 601*601
            disp("this data is problematic")
            problematicls = cat(2,problematicls, index);
            flag = 1;
        end
    else
        disp("this data is problematic")
        problematicls = cat(2,problematicls, index);
        flag = 1;
    end
    
    if flag == 0
        tocompletedatapath = toFolder+dataindex+dataappendix;
        writematrix(data_simple, tocompletedatapath)
    end
end
disp("the following images might be problematic")
disp(problematicls)